﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Data;
using System.Windows.Forms;
using System.Text.RegularExpressions;
using System.Collections;
using System.Threading;
using System.Net;
using System.IO;
using System.Diagnostics;
using System.Xml;

using ShootSearch.Core;
using ShootSearch.Common;
using ShootSearch.FileSystem;
using ShootSearch.Util;
using ShootSearch.Spiders;
using ShootSearch.Util.Serializers;

using Quartz;
using ShootSearch.Logging;


namespace ShootSearch.Jobs
{
    /// <summary>
    /// Download url delegate
    /// </summary>
    /// <param name="url">the url information</param>
    public delegate void DownloadUrlCallback(Url url);
    /// <summary>
    /// Spider job, crawl pages
    /// </summary>
    public class HttpSpiderJob : SequenceJob, IStatefulJob
    {
        #region Const
        /// <summary>
        /// Un processed queue cache path
        /// </summary>
        public const string PATH_QUEUE = "Queue";
        /// <summary>
        /// Failed queue cache path
        /// </summary>
        public const string PATH_FAILED = "Failed";
        /// <summary>
        /// Processed queue cache path
        /// </summary>
        public const string PATH_PROCESSED = "Processed";
        /// <summary>
        /// Downloaded path
        /// </summary>
        public const string PATH_DOWNLOAD = "Download";
        /// <summary>
        /// Progress file name
        /// </summary>
        public const string PATH_PROGRESS = "Progress.xml";
        /// <summary>
        /// Response stream gzip type
        /// </summary>
        public const string SPIDER_GZIP = "gzip";
        /// <summary>
        /// Response stream text based content type
        /// </summary>
        public const string SPIDET_CONTENT_TEXT = "text";
        /// <summary>
        /// Http header Date column name
        /// </summary>
        public const string SPIDER_HTTP_DATE = "Date";
        /// <summary>
        /// Http header Accept-Language column name
        /// </summary>
        public const string SPIDER_HTTP_RESPONSELENGTH = "content-length";

        public const string SPIDER_SUMMARY = "Downloading: {0}\r\n Processed:{1}, Queue:{2}, Failed:{3} ";
        #endregion

        #region Variables
        /// <summary>
        /// Configuration file
        /// </summary>
        protected HttpSpiderJobConfig config;

        /// <summary>
        /// Default encoding
        /// </summary>
        protected Encoding encodingDefault;

        /// <summary>
        /// Url parser
        /// </summary>
        protected IUrlParser urlParser;

        /// <summary>
        /// File system
        /// </summary>
        protected IFileSystem fileSaver;

        /// <summary>
        /// Failed url queue
        /// </summary>
        protected UrlQueue failed;

        /// <summary>
        /// Processed url queue
        /// </summary>
        protected UrlHashList processed;

        /// <summary>
        /// Unprocessed url queue
        /// </summary>
        protected UrlQueue queue;

        /// <summary>
        /// Update url list
        /// </summary>
        protected Dictionary<string, short> updateListDict;

        /// <summary>
        /// Running thread count
        /// </summary>
        protected int runningThread = 0;

        /// <summary>
        /// Mutex locker
        /// </summary>
        protected AutoResetEvent resetEvent = new AutoResetEvent(false);

         /// <summary>
        /// Updated url count
        /// </summary>
        protected int updatedCount;

        /// <summary>
        /// Thread locker
        /// </summary>
        protected object locker = new object();

        /// <summary>
        /// Accepted binary content type
        /// </summary>
        protected Hashtable allowContentType;

        /// <summary>
        /// Accepted binary extensions
        /// </summary>
        protected Regex regexAllowTypes;

        /// <summary>
        /// Mutiple files time switcher
        /// </summary>
        protected System.Threading.Timer fileSystemTimer;

        /// <summary>
        ///  Mutiple files size switcher
        /// </summary>
        protected long fileSystemMaxlength = 0;

        /// <summary>
        /// Spider progress saver
        /// </summary>
        protected SpiderProgress progress;

        #endregion

        #region IConfigable Memebers
        /// <summary>
        /// Spider configuration
        /// </summary>
        public override SpiderManConfig Config
        {
            get
            {
                return config;
            }
            set
            {
                if (value is HttpSpiderJobConfig)
                {
                    base.Config = value;
                    config = value as HttpSpiderJobConfig;
                }
            }
        }
        #endregion

        #region Constructor
        /// <summary>
        /// Constructor
        /// </summary>
        public HttpSpiderJob()
        {
            config = new HttpSpiderJobConfig();
            progress = new SpiderProgress();
        }
        #endregion

        #region SequenceJob Members
        /// <summary>
        /// Dowork
        /// </summary>
        /// <param name="context"></param>
        public override void DoWork(JobExecutionContext context)
        {
            if (Initialize())
            {
                LauchDownload();
                Dispose();

            }
        }

        /// <summary>
        /// override Method, add download path
        /// </summary>
        /// <param name="args"></param>
        /// <returns></returns>
        public override bool LaunchNextJob(Hashtable args)
        {
            if (args == null) args = new Hashtable();
            args.Add(InputOutputJob.INPUT_PATH, config.PathConfig.DownloadPath + PATH_DOWNLOAD);
            return base.LaunchNextJob(args);
        }

        
        #endregion
         
        #region IDisposable Members
        /// <summary>
        /// Dispose, close the file handle
        /// </summary>
        public override void Dispose()
        {
            if (fileSaver != null)
            {
                fileSaver.Close();
            }
        }

        #endregion

        #region Initialize
        /// <summary>
        /// Initialize
        /// </summary>
        protected bool Initialize()
        {
            //更新Url列表
            this.updateListDict = new Dictionary<string, short>();

            if (config.PathConfig.UsingContextPath)
            {
                config.PathConfig.DownloadPath = contextPath;
                config.PathConfig.TaskPath = contextPath;
            }

            InitializeQueues();

            //Does not support update
            if (progress.Action == SpiderAction.Updating && 
                config.InitConfig.UpdateMode == UpdateModeType.None)
            {

                return false;
            }

            InitializeEncoding();

            InitializeRegexFilters();


            //Default thread count : 5
            if (config.ThreadConfig.ThreadCount <= 0)
            {
                config.ThreadConfig.ThreadCount = 5;
            }

            //Initialize filesystem
            try
            {
                InitializeFileSystem();
                if (fileSaver == null) return false;

                //Initialize the UrlParser
                urlParser = config.FilterConfig.UrlParser.GetObjectWithConfig() as IUrlParser;
                urlParser.Logger = this.Logger;
                if (urlParser == null) return false;

            }
            catch (Exception ex)
            {
                Log.Error(Logger,"Initialize error!", ex);
                //Trace.WriteLine(e);
                return false;
                //Log.Error(e);
            }

            //Launch the spider
            if (progress.Action == SpiderAction.NotStarted)
            {
                progress.Action = SpiderAction.Processing;
            } 

            return true;
        }

        #region InitializeQueues
        /// <summary>
        /// Initialize the processed, failed & un-processed queue
        /// </summary>
        protected void InitializeQueues()
        {
            //待处理队列和更新队列
            this.queue = new UrlQueue(IO.FormatDirName(config.PathConfig.TaskPath + PATH_QUEUE));
            this.failed = new UrlQueue(IO.FormatDirName(config.PathConfig.TaskPath + PATH_FAILED));

            if (File.Exists(config.PathConfig.TaskPath + PATH_PROGRESS))
            {
                progress = SpiderProgress.Load(config.PathConfig.TaskPath + PATH_PROGRESS);
                //Load the update url list
                updateListDict = SpiderProgress.GetUpdateUrlList(progress.UpdateList);
            }
            else
            {
                //Initialize the update url list
                updateListDict = SpiderProgress.GetUpdateUrlList(config.InitConfig.UrlListUpdate);
            }


            //如果爬虫的动作是更新或者继续下载，那么需要加载已经处理过的Url队列信息
            if (progress.Action != SpiderAction.NotStarted)
            {
                this.queue.Load();
                this.failed.Load();
                this.processed = UrlHashList.Load(IO.FormatDirName(config.PathConfig.TaskPath + PATH_PROCESSED));
            }
            else
            {
                //else condition: progress.Action == SpiderAction.NotStarted
                //First time
                //Initialize the queue config
                this.queue.BlockLength = this.config.UrlQueueConfig.UrlBlockLength;
                this.failed.BlockLength = this.config.UrlQueueConfig.UrlBlockLength;
                this.processed = new UrlHashList(this.config.UrlQueueConfig.UrlScale);
                this.processed.CachePath = IO.FormatDirName(config.PathConfig.TaskPath + PATH_PROCESSED);

                InitializeStartUrls();

            }
        } 
        #endregion

        #region Initialize FileSystem
        /// <summary>
        /// Initialize the FileSystem
        /// </summary>
        protected void InitializeFileSystem()
        {
            //Load filesystem
            this.fileSaver = (IFileSystem)config.SaveConfig.FileSystem.GetObjectWithConfig();

            if ((fileSaver != null))
            {
                fileSaver.Logger = this.Logger;
                fileSaver.FSID = progress.FileSystemID.ToString(config.SaveConfig.FileSystemIDFormat);
                if (progress.Action == SpiderAction.Updating)
                {
                    SwitchFSID(false);
                }

                fileSaver.CreateOpen(IO.FormatDirName(config.PathConfig.DownloadPath + PATH_DOWNLOAD));

                if (config.SaveConfig.EanbleMultiFileSystem)
                {
                    if (config.SaveConfig.MultiFileSystemType == MultiFileSystemType.BySzie)
                    {
                        fileSaver.FileSystemWrite += new FileSystemActionEventHandler(FileSaver_FileSystemWrite);
                        fileSystemMaxlength = config.SaveConfig.MultiFileSystemParameter * 1024 * 1024;
                    }
                    else
                    {
                        fileSystemTimer = new System.Threading.Timer(new TimerCallback(FileSystemChangeIDTimer));
                        fileSystemTimer.Change(config.SaveConfig.MultiFileSystemParameter * 60 * 1000, 0);
                    }
                    //fileSystemID = int.Parse(fileSaver.FSID);
                }
            }
        } 
        #endregion

        #region Initialize the Regex Filters
        protected void InitializeRegexFilters()
        {      
            //Initialize the extension and the ContentType
            regexAllowTypes = Tools.CreateRegex(config.FilterConfig.ContactToRegex());
            allowContentType = new Hashtable();
            if (config.FilterConfig.FileTypes != null)
            {
                foreach (BinFileDescType filetype in config.FilterConfig.FileTypes)
                {
                    if (!allowContentType.Contains(filetype.ContentType))
                    {
                        allowContentType.Add(filetype.ContentType, null);
                    }
                }
            }
        }
        
        #endregion

        #region Initialize Encoding
        /// <summary>
        /// Initialize Encoding
        /// </summary>
        protected void InitializeEncoding()
        {
            //Initialize the encoding
            if (this.config.InitConfig.Encode.ToLower().EndsWith("default"))
            {
                this.encodingDefault = Encoding.Default;
            }
            else
            {
                if (config.InitConfig.Encode.ToLower().Trim() == "auto")
                {
                    this.encodingDefault = null;
                }
                else
                {
                    try
                    {
                        this.encodingDefault = Encoding.GetEncoding(config.InitConfig.Encode);
                    }
                    catch (Exception ex)
                    {
                        Log.Warn(Logger,"InitializeEncoding Error! ", ex);
                        //Trace.WriteLine(ex);
                        this.encodingDefault = Encoding.Default;
                    }
                }
            }
        } 
        #endregion

        #region Initialize start Urls
        /// <summary>
        /// Load start Urls form the configuration
        /// </summary>
        protected void InitializeStartUrls()
        {
            //Initizlize the start urls
            if (config.InitConfig.StartListAsFilePath)
            {
                foreach (string item in config.InitConfig.UrlListStart)
                {
                    if (File.Exists(item))
                    {
                        string[] startUrls = File.ReadAllLines(item);
                        foreach (string urlItem in startUrls)
                        {
                            if (!string.IsNullOrEmpty(urlItem.Trim()))
                            {
                                this.queue.EnQueue(new Url(urlItem.Trim()));
                            }
                        }
                    }

                }
            }
            else
            {
                foreach (string item in config.InitConfig.UrlListStart)
                {
                    this.queue.EnQueue(new Url(item));
                }
            }
        }

        #endregion

        #endregion

        #region LaunchDownload
        /// <summary>
        /// Process unprocessed queue
        /// </summary>
        protected void LauchDownload()
        {
            DownloadUrlCallback downloadUrlCallback = new DownloadUrlCallback(DownloadUrl);

            while (Status == JobStatus.Running)
            {
                if (runningThread == 0 && queue.Length == 0)
                {
                    Status = JobStatus.Completing;
                }
                else if (runningThread > 0 && queue.Length == 0)
                {
                    //Wait if there are some thread is running
                    Thread.Sleep(1000);
                }
                else
                {
                    //Up to max thread count, wait until some thread complete their work
                    if (runningThread >= config.ThreadConfig.ThreadCount)
                    {
                        resetEvent.WaitOne();
                    }

                    //TODO: Complete the update mode
                    Url url = queue.DeQueue();
                    if (!processed.Contains(url.Address))
                    {
                        processed.Add(url);
                        runningThread++;
                        downloadUrlCallback.BeginInvoke(url, null, downloadUrlCallback);
                    }
                }
            }

            while (runningThread > 0)
            {
                Thread.Sleep(1000);
            }

            SaveQueue();

            if (Status == JobStatus.Completing)
            {
                Log.Info(Logger, "Completed!");
                StatusSummary = "Completed";
            }
            else
            {
                Log.Info(Logger, "Stopped!");
                StatusSummary = "Stopped";
            }


        }


        #endregion

        #region Switch FSID
        /// <summary>
        /// Switch the FSID by file length
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        protected void FileSaver_FileSystemWrite(object sender, ShootSearch.FileSystem.FileSystemEventArgs e)
        {
            if (fileSaver.Length > fileSystemMaxlength)
            {
                SwitchFSID(true);
            }
        }

        /// <summary>
        /// Switch the FSID by Time
        /// </summary>
        /// <param name="obj"></param>
        protected void FileSystemChangeIDTimer(object obj)
        {
            SwitchFSID(true);
        }

        protected void SwitchFSID(bool canLaunchNextJob)
        {
            string fsid = fileSaver.FSID;
            fileSaver.FSID = (++progress.FileSystemID).ToString(config.SaveConfig.FileSystemIDFormat);

            if (canLaunchNextJob && config.LaunchNextJob == LaunchNextJobType.Manually)
                LaunchNextJob(fsid);

        }

        protected void LaunchNextJob(string fsid)
        {
            Hashtable args = new Hashtable();
            args.Add(InputOutputJob.INPUT_FSID, fsid);
            LaunchNextJob(args);           
        }
        #endregion

        #region DownloadUrl
        /// <summary>
        /// Download a URL
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        protected void DownloadUrl(Url url)
        {
   
            string page = string.Empty;
            
            Uri uri = null;
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            Stream stream = null;

            //Prevent the uri constructor error
            if (Uri.TryCreate(url.Address, UriKind.Absolute, out uri))
            {
                try
                {
                    #region Create HttpWebRequest
                    //Create HttpWebRequest
                    request = WebRequest.Create(uri) as HttpWebRequest;
                    //Setup header
                    request.Accept = config.ThreadConfig.HttpAccept;
                    request.Method = config.ThreadConfig.HttpMethod;
                    request.Referer = config.ThreadConfig.HttpReferer;
                    request.KeepAlive = true;
                    request.UserAgent = config.ThreadConfig.HttpUserAgent;
                    for (int i = 0; i < config.ThreadConfig.HttpHeaders.Count; i++)
                    {
                        HttpHeader header = config.ThreadConfig.HttpHeaders[i];
                        try
                        {
                            request.Headers[header.Name] = header.Value;
                        }
                        catch (Exception exHeader)
                        {
                            Log.Warn(Logger, "Header setup error: " + header.Name, exHeader);
                            //Once meet the header setup error, remove the err header
                            lock (config.ThreadConfig.HttpHeaders)
                            {
                                config.ThreadConfig.HttpHeaders.RemoveAt(i);
                                if (config.ThreadConfig.HttpHeaders.Count == 0) break;
                            }
                        }
                    }

                    //request.Headers["Accept-Language"] = config.ThreadConfig.HttpAcceptLanguage;
                    //request.Headers["Accept-Encoding"] = config.ThreadConfig.HttpAcceptEncoding;
                    //request.Headers["Accept-Charset"] = config.ThreadConfig.HttpAcceptCharset;
                    request.Timeout = config.ThreadConfig.TimeOut * 1000;
                    #endregion

                    //Send request
                    response = request.GetResponse() as HttpWebResponse;

                    if (response != null)
                        stream = response.GetResponseStream();

                    //Check response stream length
                    if (CheckLength(response, stream))
                    {
                        #region Check content type
                        //Accept content type:
                        //1. empty 2. text 3. accept binary types 4. accept binary extensions
                        if (string.IsNullOrEmpty(response.ContentType) ||
                                response.ContentType.StartsWith(SPIDET_CONTENT_TEXT) ||
                                allowContentType.Contains(response.ContentType.ToLower()) ||
                                (regexAllowTypes != null && regexAllowTypes.IsMatch(url.Address)))
                        {
                            Log.Info(Logger, "Downloading[" + Thread.CurrentThread.ManagedThreadId + "]: " + url.Address);
                            StatusSummary = string.Format(SPIDER_SUMMARY, url.Address, processed.Length
                                , queue.Length, failed.Length);

                            //Read stream into memory
                            MemoryStream memStream = new MemoryStream();
                            IO.Pump(stream, memStream, config.SaveConfig.BufferChunk);
                            memStream.Seek(0, SeekOrigin.Begin);
                            //Close the net stream
                            stream.Close();

                            #region Read the stream in  to bytes
                            //Check if the stream is gzip type
                            byte[] bytes = new byte[0];
                            if (response.ContentEncoding.ToLower() == SPIDER_GZIP)
                            {
                                MemoryStream memOriginalStream = new MemoryStream();
                                IO.Decompress(memStream, memOriginalStream);
                                //Get byte array
                                bytes = new byte[(int)memOriginalStream.Length];
                                memOriginalStream.Seek(0, SeekOrigin.Begin);
                                memOriginalStream.Read(bytes, 0, bytes.Length);

                            }
                            else
                            {
                                //Get byte array
                                bytes = new byte[(int)memStream.Length];
                                memStream.Read(bytes, 0, bytes.Length);
                            }
                            #endregion

                            #region Parse page & save the content
                            IFileObject obj = new FileObject();
                            obj.Url = url.Address;
                            DateTime dt = DateTime.Now;
                            //Page data time
                            if (DateTime.TryParse(response.Headers[SPIDER_HTTP_DATE], out dt))
                            {
                                obj.DateTime = dt;
                            }
                            else
                            {
                                obj.DateTime = DateTime.Now;
                            }
                            obj.ContentType = WebPage.ParseContentType(response.ContentType);
                            obj.Encoding = Encoding.UTF8;

                            //Check the content type & save the content
                            if (response.ContentType.StartsWith(SPIDET_CONTENT_TEXT))
                            {
                                Encoding encoding = WebPage.GetEncoding(response.ContentType, null);
                                if (encoding == null) encoding = WebPage.TestPageEncoding(bytes, encodingDefault);
                                page = encoding.GetString(bytes);
                                //Set text content
                                obj.Content = Encoding.UTF8.GetBytes(page);
                                //Parse the page links
                                ParsePage(url, page);
                            }
                            else
                            {
                                //Set binary content
                                obj.Content = bytes;
                            }
                            //Save the content
                            fileSaver.Write(obj);
                            #endregion
                        }
                        #endregion
                    }
                    //Close the response object
                    response.Close();
                }
                catch (Exception ex)
                {
                    Log.Warn(Logger, "Download error: " + url.Address, ex);
                    failed.EnQueue(url);
                }
            }
            //Release the thread
            runningThread--;
            resetEvent.Set();
        }
        
        #endregion

        #region CheckLength
        /// <summary>
        /// Check file stream length
        /// </summary>
        /// <param name="response">Response object</param>
        /// <param name="stream">Response.ResponseStream</param>
        /// <returns></returns>
        protected bool CheckLength(HttpWebResponse response, Stream stream)
        {
            //Check empty stream and maximum or minimum file length has value
            bool success = stream != null;
            if (!success ||
                config.FilterConfig.MaxFileLength == 0 && config.FilterConfig.MinFileLength == 0) return success;

            //Try to read the length in reaponse header
            long flength = -1;

            try
            {
                flength = Convert.ToInt64(response.Headers[SPIDER_HTTP_RESPONSELENGTH]);
            }
            catch (FormatException ex)
            {
                Log.Warn(Logger, "content-length error: " + response.ResponseUri, ex);
                //System.Diagnostics.Debug.WriteLine(ex.Message);
                flength = -1;
            }

            //Check the file length
            if (flength != -1)
            {
                if ((config.FilterConfig.MaxFileLength > 0)
                    && (flength > config.FilterConfig.MaxFileLength))
                {
                    success = false;
                }
                if ((config.FilterConfig.MinFileLength > 0)
                    && (flength < config.FilterConfig.MaxFileLength))
                {
                    success = false;
                }
            }

            return success;
        }
        #endregion

        #region ParsePage
        /// <summary>
        /// Parse the links of the page
        /// </summary>
        protected void ParsePage(Url url, string page)
        {
            //Can be parsed
            if (config.FilterConfig.UrlMaxDepth < 0
                || url.Depth < config.FilterConfig.UrlMaxDepth)
            {
                List<Url> urls = urlParser.ParsePage(url, page);
                foreach (Url item in urls)
                {
                    //Update update urls
                    if (config.FilterConfig.UrlUpdateRule != null && config.FilterConfig.UrlUpdateRule.IsMatch(item.Address)
                        && !updateListDict.ContainsKey(item.Address))
                    {
                        updateListDict.Add(item.Address, item.Depth);
                    }

                    queue.EnQueue(item);
                }
            }
        }
        #endregion

        #region SaveQueue
        /// <summary>
        /// Save the queue
        /// </summary>
        protected void SaveQueue()
        {
            if (Status == JobStatus.Completing || Status == JobStatus.Completed)
            {
                progress.Action = SpiderAction.Updating;
            }

            //Using locker, could be saved while running
            lock (locker)
            {                
                //Process Update Url List
                progress.UpdateList = SpiderProgress.UpdateUrlList(updateListDict);
                //Save progress
                this.progress.Save(contextPath + PATH_PROGRESS);
                //Save queue
                this.queue.Save();
                //Save failed
                this.failed.Save();
                //Save processed urls
                this.processed.Save();

                fileSaver.Flush();

                Log.Info(Logger, "Save Queue");
            }
        }
        #endregion

    }
    
}
